{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection\n",
    "from cot.create import unique_index_correction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## correct existing files for unique ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from collections import Counter\n",
    "\n",
    "# item_counts = Counter(id_list)\n",
    "\n",
    "# # Use list comprehension to create a new list with items that have a count > 1\n",
    "# duplicates = [item for item, count in item_counts.items() if count > 1]\n",
    "\n",
    "# print(duplicates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = Collection.from_json(\"/home/kon/CoT/ThoughtSource/libs/cot/tests/unit_tests/data/worldtree_100_dataset.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "311a4f681970441e9dd51e191edff31c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "coll = unique_index_correction(coll)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d74c981b975049f1b88f9c193aa2d10f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "coll.dump(\"/home/kon/CoT/ThoughtSource/libs/cot/tests/unit_tests/data/worldtree_100_dataset_corrected.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fe7b207eefe4bf0a6468338be839032",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d9ff63caad84f18b700549a532d2b14",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3f76c1ac0c3400d9c7fe5315bd93a73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ts_100 = unique_index_correction(ts_100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe115acfc8614fe0b671cddc1977f1cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "996c6ab647b249cc81473b569d8cc6c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54846b58dc2044a3a35a606e2426f4bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3229ae8d8fe5442789d96111070de790",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67017b7a88ff469abf401b61828f1df4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36bebccae5a84e3688dd79d33def7b55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ts_100.dump(\"thoughtsource_100\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply the same function unique_index_correction too all json files in the current directory\n",
    "dir = \"./\"\n",
    "import os\n",
    "from cot import Collection\n",
    "filenames = os.listdir(dir)\n",
    "filenames = [filename for filename in filenames if filename.endswith(\".json\")]\n",
    "filenames = sorted(filenames)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['chat-gpt_vs_gpt4_none_cot.json',\n",
       " 'chat-gpt_vs_gpt4_only_zhou-cot.json',\n",
       " 'chat-gpt_vs_gpt4_vs_t5_zhou-and-none_cot.json',\n",
       " 'chatgpt_hard_dataset.json',\n",
       " 'chatgpt_hard_dataset_IQI_reasoning.json',\n",
       " 'chatgpt_hard_dataset_IQI_respone_requirements.json',\n",
       " 'chatgpt_hard_dataset_qa-01_to_qa-09_zhou-01-ins.json',\n",
       " 'chatgpt_hard_dataset_qa-01_to_qa-09_zhou-01-ins_zhou-01.json',\n",
       " 'chatgpt_hard_dataset_qa-10_to_qa-19.json',\n",
       " 'current.json',\n",
       " 'filtered_thoughtsource_100_gpt-4_None_false.json',\n",
       " 'filtered_thoughtsource_100_gpt-4_false.json',\n",
       " 'filtered_thoughtsource_100_gpt-4_false_only_gpt-4_cots.json',\n",
       " 'filtered_ts_100_gpt-4_None_correct_zhou-01_false.json',\n",
       " 'filtered_ts_100_gpt-4_None_false_zhou-01_correct.json',\n",
       " 'foo_save_2.json',\n",
       " 'foo_select_answers.json',\n",
       " 'gpt-3.5_None_and_zhou-01_vs_gpt-4_None_and_zhou-01.json',\n",
       " 'gpt-3.5_None_and_zhou-01_vs_gpt-4_zhou-01.json',\n",
       " 'gpt-4_None.json',\n",
       " 'gpt-4_zhou-01.json',\n",
       " 'qa-05-06-07.json',\n",
       " 'qa-05-06.json',\n",
       " 'qa-05.json',\n",
       " 'qa-06.json',\n",
       " 'qa-07.json',\n",
       " 'qa-08.json',\n",
       " 'qa-09.json',\n",
       " 'save_result_to_be_deleted.json',\n",
       " 'thoughtsource_100_empty.json',\n",
       " 'thoughtsource_1_selected.json',\n",
       " 'ts-01-cot.json',\n",
       " 'ts_1_sel_gen.json',\n",
       " 'ts_20_random.json',\n",
       " 'ts_hard_v1.json',\n",
       " 'ts_hard_v1_ts_cots.json',\n",
       " 'ts_hard_v2.json',\n",
       " 'ts_hard_v3.json',\n",
       " 'worldtree_10.json']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for filename in filenames:\n",
    "    print(filename)\n",
    "    collection = Collection.from_json(os.path.join(dir, filename))\n",
    "    collection = unique_index_correction(collection)\n",
    "    collection.dump(os.path.join(dir, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_to_all_json_files_in_dir('./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ts_hard_v1.json',\n",
       " 'results_presentation.ipynb',\n",
       " 'qa-08.json',\n",
       " 'ts_hard_v1_ts_cots.json',\n",
       " 'chatgpt_hard_dataset_IQI_respone_requirements.json',\n",
       " 'chat-gpt_vs_gpt4_only_zhou-cot.json',\n",
       " 'current.json',\n",
       " 'qa-09.json',\n",
       " 'tutorial.ipynb',\n",
       " 'gpt-4_zhou-01.json',\n",
       " 'chat-gpt_vs_gpt4_vs_t5_zhou-and-none_cot.json',\n",
       " 'foo_select_answers.json',\n",
       " '__pycache__',\n",
       " 'ts_hard_v3.json',\n",
       " 'foo_save_2.json',\n",
       " 'gpt-3.5_None_and_zhou-01_vs_gpt-4_None_and_zhou-01.json',\n",
       " 'ts-01-cot.json',\n",
       " 'instruction_experiments.ipynb',\n",
       " 'thoughtsource_100_empty.json',\n",
       " 'chatgpt_hard_dataset_qa-10_to_qa-19.json',\n",
       " 'instruction_experiments_runs.ipynb',\n",
       " 'filtered_ts_100_gpt-4_None_false_zhou-01_correct.json',\n",
       " 'item_selection.ipynb',\n",
       " 'ChatGpt-vs-gpt4',\n",
       " 'filtered_thoughtsource_100_gpt-4_false.json',\n",
       " 'thoughtsource_100_T5.xlsx',\n",
       " 'qa-05-06.json',\n",
       " 'qa-05.json',\n",
       " 'chatgpt_hard_dataset_qa-01_to_qa-09_zhou-01-ins.json',\n",
       " 'qa-07.json',\n",
       " 'worldtree_10.json',\n",
       " 'ts_20_random.json',\n",
       " 'chatgpt_hard_dataset_qa-01_to_qa-09_zhou-01-ins_zhou-01.json',\n",
       " 'ts_1_sel_gen.json',\n",
       " 'thoughtsource_1_selected.json',\n",
       " 'qa-06.json',\n",
       " 'internal_documentation',\n",
       " 'gpt-4_None.json',\n",
       " 'filtered_thoughtsource_100_gpt-4_false_only_gpt-4_cots.json',\n",
       " 'chatgpt_hard_dataset_IQI_reasoning.json',\n",
       " 'ts_hard_v2.json',\n",
       " 'cot_saving.ipynb',\n",
       " 'qa-05-06-07.json',\n",
       " 'filtered_thoughtsource_100_gpt-4_None_false.json',\n",
       " 'thoughtsource_100.xlsx',\n",
       " 'gpt-3.5_None_and_zhou-01_vs_gpt-4_zhou-01.json',\n",
       " 'chat-gpt_vs_gpt4_none_cot.json',\n",
       " 'chatgpt_hard_dataset.json',\n",
       " 'datasets_analysis.ipynb',\n",
       " 'filtered_ts_100_gpt-4_None_correct_zhou-01_false.json',\n",
       " 'save_result_to_be_deleted.json']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "os.listdir('./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/kon/work/ThoughtSource/notebooks\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a19cb823e24c98ee05a9cfa4a3a579b5d56d4c1a735f2a12456750b95a1e155e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
